from typing import Iterable
import scrapy
from scrapy.http import HtmlResponse, Request
from doubanScrapy.items import MovieItem

# 定义豆瓣电影 Top 250 的爬虫程式, scrapy crawl doubanSpider -o douban.csv
class DoubanspiderSpider(scrapy.Spider):
    name = "doubanSpider"
    allowed_domains = ["movie.douban.com"]
    
    # 静态指定爬虫页面（二选一）
    start_urls = ['https://movie.douban.com/top250']
    # 动态指定爬虫页面（二选一）
    # def start_requests(self) -> Iterable[Request]:
    #     for page in range(10):
    #         yield scrapy.Request(url=f'https://movie.douban.com/top250?start={page * 25}&filter=')

    # 解析网页内容
    def parse(self, response: HtmlResponse, **kwargs):
        sel = scrapy.Selector(response)
        list_items = sel.css('#content > div > div.article > ol > li')
        for list_item in list_items:
            detail_url = list_item.css('div.info > div.hd > a::attr(href)').extract_first()
            movie_item = MovieItem()
            movie_item['title'] = list_item.css('span.title::text').extract_first()
            movie_item['rank'] = list_item.css('span.rating_num::text').extract_first()
            movie_item['subject'] = list_item.css('span.inq::text').extract_first()
            # 爬取下一层电影细节内容，取得'播放时长'和'电影简介'
            yield scrapy.Request(
                url = detail_url,
                callback = self.parse_detail,
                cb_kwargs = {'item': movie_item}
            )

    # 解析电影细节内容
    def parse_detail(self, response: HtmlResponse, **kwargs):
        movie_item = kwargs["item"]
        sel = scrapy.Selector(response)
        movie_item['duration'] = sel.css('span[property="v:runtime"]::attr(content)').extract_first()
        movie_item['intro'] = sel.css('span[property="v:summary"]::text').extract_first()
        yield movie_item